#include	<algorithm>
#include	"utf16.h"
#include	<stdint.h>
#include	<iostream>
#include	<sys/mman.h>
#include	<sys/stat.h>
#include	<sys/time.h>
#include	<unistd.h>
#include	<cstdlib>
#include	<fcntl.h>
#include	<fstream>
#include	<vector>
#include	<cmath>
using	namespace	std;
struct	Sort {
    uint16_t	i;
    uint64_t	n;
    bool	operator()(Sort	X,	Sort	Y) {
        return	X.i==0?true:(Y.i==0?false:X.n>Y.n);
    }
};
int	main(int	ac,	char	**av) {
    if(ac<3) {
        cerr<<"voca input task(task=0: utf8 to utf16; task=1: generate voca.txt)\n";
        return	0;
    }
    if(atoi(av[2])==0) {
        string	line;
        vector<uint16_t>	u;
        ifstream	fi(av[1]);
        string	fn=av[1];
        fn+=".utf16";
        FILE	*f=fopen(fn.c_str(),"wb");
        while(getline(fi,line)) {
            line.push_back('\n');
            u.resize(line.size());
            size_t	n=utf8_to_utf16((uint8_t*)line.data(),line.size(),u.data(),u.size());
            fwrite(u.data(),n*2,1,f);
        }
        fclose(f);
        fi.close();
    } else {
        string	fn=av[1];
        fn+=".utf16";
        uint16_t	*ptr;
        int	fd;
        struct	stat	sb;
        fd=open(fn.c_str(),	O_RDONLY);
        fstat(fd,	&sb);
        ptr=(uint16_t*)mmap(NULL,	sb.st_size,	PROT_READ,	MAP_SHARED,	fd,	0);
        uint64_t	tot=sb.st_size>>1;
        Sort	vs[65536];
        for(size_t	i=0;	i<65536;	i++) {
            vs[i].i=i;
            vs[i].n=0;
        }
        for(uint64_t	i=0;	i<tot;	i++)	vs[ptr[i]].n++;
        munmap(ptr,sb.st_size);
        close(fd);

        double	ent=0;
        for(uint64_t	i=0;	i<65536;	i++)	if(vs[i].n) {
                double	p=(double)vs[i].n/tot;
                ent+=p*log(p);
            }
        cerr<<"prior:\t"<<exp(ent)<<'\n';
        sort(vs,vs+65536,Sort());
        ofstream	fo("voca.txt");
        uint64_t n=0,	id=0,	suggest=0;
        for(size_t	i=0;	i<65536;	i++) if(vs[i].n||vs[i].i==0) {
                n+=vs[i].n;
                id++;
                fo<<vs[i].i<<"\n";
                if(id%16==0&&n>=0.9999*tot&&suggest==0)	suggest=id;
            }
        fo.close();
        cerr<<"suggested voca:\t"<<suggest<<'\n';
    }
    return	0;
}
